# -*- coding: UTF-8 -*-
# cython: language_level=3
import requests
from concurrent import futures
from lxml.html import fromstring
from retry import retry
from tqdm import tqdm

from db.mongodb import MongoDB
from db.proxy_db import ProxyDB
from utils.own_tools import get_host

db = MongoDB()

headers = {
    'authority': 'yandex.com',
    'pragma': 'no-cache',
    'cache-control': 'no-cache',
    'device-memory': '4',
    'dpr': '1',
    'viewport-width': '1920',
    'rtt': '300',
    'downlink': '1.3',
    'ect': '3g',
    'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Microsoft Edge";v="99"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'upgrade-insecure-requests': '1',
    'dnt': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.36',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-user': '?1',
    'sec-fetch-dest': 'document',
    'referer': 'https://yandex.com/',
    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    # 'cookie': 'mda=0; yandex_gid=10590; yandexuid=4304287981646883686; yuidss=4304287981646883686; is_gdpr=0; is_gdpr_b=CPrlYRDOZigC; i=ZNXQaB8T4rjZpgCCBtMDHbC3v+mI5qHJMRaYrSl9QzZOzqCuSX8DJZO2FOZXTYBdQk5Z5MJ0w9tlFrpJPnwvzO2U6+U=; my=YwA=; gdpr=0; _ym_uid=1646883687489598949; _ym_isad=1; involved=1; spravka=dD0xNjQ2ODg0Mjc4O2k9MjQwODo4NDA2OmNlMDo5MjVlOjY1NTE6ZjJhMToyNjJiOmM2OGQ7RD1DNzA5OEZERjNENTlENEVCNTVCQUE5MzY5NUZCNEFDOUU1MTAzMThBMDY4QjUzRTczQkYxMDVCNTA5NjdCN0IxQzY3RTQxODM7dT0xNjQ2ODg0Mjc4NzI1ODg3MTA1O2g9ZTc4NWRhYTk2YTc4OTViMzBkYTUzMzkyNmFiZDY3NTk=; _ym_d=1646884279; _ym_visorc=b; ys=wprid.1646885101466735-10269918504242332149-man1-7387-man-l7-balancer-8080-BAL-2100; _yasc=NLfY+gHcxpWJ3hADqMEbOQ3YhOjIRvAuMARVKK0HRHTrCu5EouANLAVlOZcTuQ==; yp=1649475687.ygu.1#1647488593.szm.1:1920x1080:990x556#1646970194.ln_tp.01#1678420301.sp.family%3A0',
}


@retry(tries=30)
def search(keyword, page=0):
    search_url = f"https://yandex.com/search/?text={keyword}&p={page}&lr=10596"
    print(search_url)
    proxy = ProxyDB().get_proxy()
    proxy = {'http': proxy, 'https': proxy} if proxy else None
    response = requests.get(search_url, proxies=proxy, headers=headers, timeout=(3, 10))
    # print(response.text)
    response.raise_for_status()
    html = fromstring(response.text)
    l = html.xpath('//div[@class="content__left"]/ul//li')
    assert l, Exception('ban ip')
    for li in l:
        # print(tostring(li))
        try:
            title = ''.join(li.xpath('.//h2//text()')).strip()
            url = li.xpath('.//a[@tabindex="0"]/@href')[0]
            print(title, url)
            h = get_host(url)
            # h['title'] = title
            if not db.add('INPUT_URLS', {"url": h['home_page']}):
                print('exist')
        except:
            pass


def main():
    tasks, results = [], []

    with futures.ThreadPoolExecutor(max_workers=80) as executor:
        city_keywords = ['粤icp', '广州', '', '羊城', '越秀区', '荔湾区', '海珠区', '天河区', '白云区', '黄埔区', '番禺区', '花都区', '南沙区', '增城区',
                         '从化区']
        # releative_keywords = ['动漫', '动画', '教学视频，视频，电影天堂，电视剧，纪录片', '短视频', '微视频', '微电影', '教育视频', '新闻视频', '音乐视频', '舞蹈视频',
        #                       '竞技视频', '比赛视频', '体育视频', '娱乐视频', '在线播放', '在线观看，正在播放',
        #     '视频', '电影', '电视剧', '综艺', '纪录片', '动画', '动漫', '国漫', '日漫',
        #     '流行首播', '热播', '在线播放', '在线观看', '正在播放',
        #     '大陆', '港台', '欧美', '日韩', '美剧', '韩剧', '华语', '热舞', '嘻哈', '摇滚',
        #     'dytt', '电影天堂'
        # ]
        releative_keywords = ['比赛', '综艺', '日漫', 'dytt', '动漫', '音乐', '微电影', '在线', '欧美', '舞蹈', '观看', '新闻', '日韩', '播放',
                              '动画',
                              '热舞',
                              '电影', '国漫', '电视剧', '港台', '韩剧', '教育', '视频 ', '嘻哈', '华语', '纪录片', '大陆', '体育', '美剧', '摇滚',
                              '首播',
                              '娱乐',
                              '热播', '电影天堂',
                              '1024', '4480yy', '6080', '91', '99', 'AV', 'Av', 'HEZYO', 'aV', 'ag777', 'av', 'bbin',
                              'porm',
                              'pron', '一本道', '一道', '丁香', '下马', '不卡', '不要', '东京', '丝瓜', '丝袜', '久久', '久草', '乱伦', '五月天',
                              '人人爱',
                              '人妇', '人妻', '人成', '伊人', '作爱', '做受', '做爱', '偷偷', '偷拍', '偷窥', '兽兽', '凌辱', '加勒比', '口交', '口工',
                              '同居',
                              '啪啪', '喷水', '四虎', '夜夜', '女厕', '奶子', '奶罩', '好色', '婷婷', '射死', '小姐', '尹人', '巨乳', '强奷', '很鲁视',
                              '快猫',
                              '性交', '性爱', '性生活', '情色', '成人版', '成版人', '我爱大', '操逼', '无码', '日日夜夜', '最污', '杳蕉', '毛片', '波多野',
                              '熟妇',
                              '爱色', '特黄', '狠狠', '男朋友', '私处', '级片', '结衣', '缴情', '自偷', '自慰', '自拍', '自那慰', '色区', '色原',
                              '色噜噜',
                              '色情',
                              '色网', '色色', '草网', '菠萝', '萝莉', '衬衫', '要色', '超碰', '轮奸', '青娱乐', '青青草', '韩私', '高潮', '鲁一鲁',
                              '鲁啪',
                              '黄片',
                              '黄瓜', '黄短', '黄色录像', '黄色片', '黄色电影', '黄频']

        print("-----start-----")
        # city_keywords.reverse()
        for city_keyword in city_keywords:
            for releative_keyword in releative_keywords:
                keywords = releative_keyword + ' ' + city_keyword
                for i in range(30):
                    tasks.append(executor.submit(search, keywords, i))
        for task in tqdm(futures.as_completed(tasks), total=len(tasks), desc='yandex'):
            # results.append(task.result())
            pass


if __name__ == '__main__':
    # srarch('小说', page=0)
    main()
